<html>
 <head>
  <meta charset="utf-8"/>
  <meta content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no" name="viewport"/>
  <title>
   Hadoop 平台问题解决思路集合  | 数螺 | NAUT IDEA
  </title>
  <link href="http://cdn.bootcss.com/bootstrap/3.3.6/css/bootstrap-theme.min.css" rel="stylesheet"/>
  <link href="http://cdn.bootcss.com/bootstrap/3.3.6/css/bootstrap.min.css" rel="stylesheet"/>
  <style type="text/css">
   #xmain img {
                  max-width: 100%;
                  display: block;
                  margin-top: 10px;
                  margin-bottom: 10px;
                }

                #xmain p {
                    line-height:150%;
                    font-size: 16px;
                    margin-top: 20px;
                }

                #xmain h2 {
                    font-size: 24px;
                }

                #xmain h3 {
                    font-size: 20px;
                }

                #xmain h4 {
                    font-size: 18px;
                }


                .header {
	           background-color: #0099ff;
	           color: #ffffff;
	           margin-bottom: 20px;
	        }

	        .header p {
                  margin: 0px;
                  padding: 10px 0;
                  display: inline-block;  
                  vertical-align: middle;
                  font-size: 16px;
               }

               .header a {
                 color: white;
               }

              .header img {
                 height: 25px;
              }
  </style>
  <script src="http://cdn.bootcss.com/jquery/3.0.0/jquery.min.js">
  </script>
  <script src="http://nautstatic-10007657.file.myqcloud.com/static/css/readability.min.js" type="text/javascript">
  </script>
  <script type="text/javascript">
   $(document).ready(function() {
                 var loc = document.location;
                 var uri = {
                  spec: "http://dataunion.org/22887.html",
                  host: "http://dataunion.org",
                  prePath: "http://dataunion.org",
                  scheme: "http",
                  pathBase: "http://dataunion.org/"
                 };
    
                 var documentClone = document.cloneNode(true);
                 var article = new Readability(uri, documentClone).parse();
     
                 document.getElementById("xmain").innerHTML = article.content;
                });
  </script>
  <!-- 1466460677: Accept with keywords: (title(0.285714285714):社区,平台,Hadoop,数盟,问题,集合,思路, topn(0.3):社区,路径,Hadoop,信息,行业资讯,任务,内存,官网,目录,标签,hadoop,硬件,参数,文章,服务器,日志,数据,节点,数盟,问题,地址,spark,权限,资源管理,资源,编程语言,集群,数据量,思路,资源分配).-->
 </head>
 <body onload="">
  <div class="header">
   <div class="container">
    <div class="row">
     <div class="col-xs-6 col-sm-6 text-left">
      <a href="/databee">
       <img src="http://nautidea-10007657.cos.myqcloud.com/logo_white.png"/>
      </a>
      <a href="/databee">
       <p>
        数螺
       </p>
      </a>
     </div>
     <div class="hidden-xs col-sm-6 text-right">
      <p>
       致力于数据科学的推广和知识传播
      </p>
     </div>
    </div>
   </div>
  </div>
  <div class="container text-center">
   <h1>
    Hadoop 平台问题解决思路集合
   </h1>
  </div>
  <div class="container" id="xmain">
   ﻿﻿
   <title>
    Hadoop 平台问题解决思路集合 | 数盟社区
   </title>
   <!-- All in One SEO Pack 2.2.7.6.2 by Michael Torbert of Semper Fi Web Design[32,64] -->
   <!-- /all in one seo pack -->
   <!--
<div align="center">
<a href="http://strata.oreilly.com.cn/hadoop-big-data-cn?cmp=mp-data-confreg-home-stcn16_dataunion_pc" target="_blank"><img src="http://dataunion.org/wp-content/uploads/2016/05/stratabj.jpg"/ ></a>
</div>
-->
   <header id="header-web">
    <div class="header-main">
     <hgroup class="logo">
      <h1>
       <a href="http://dataunion.org/" rel="home" title="数盟社区">
        <img src="http://dataunion.org/wp-content/themes/yzipi/images/logo.png"/>
       </a>
      </h1>
     </hgroup>
     <!--logo-->
     <nav class="header-nav">
      <ul class="menu" id="menu-%e4%b8%bb%e8%8f%9c%e5%8d%95">
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-has-children menu-item-71" id="menu-item-71">
        <a href="http://dataunion.org/category/events" title="events">
         活动
        </a>
        <ul class="sub-menu">
         <li class="menu-item menu-item-type-post_type menu-item-object-page menu-item-22457" id="menu-item-22457">
          <a href="http://dataunion.org/2016timeline">
           2016档期
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-22459" id="menu-item-22459">
          <a href="http://dataunion.org/category/parterc">
           合作会议
          </a>
         </li>
        </ul>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category current-post-ancestor menu-item-has-children menu-item-20869" id="menu-item-20869">
        <a href="http://dataunion.org/category/tech" title="articles">
         文章
        </a>
        <ul class="sub-menu">
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-20867" id="menu-item-20867">
          <a href="http://dataunion.org/category/tech/base" title="base">
           基础架构
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-3302" id="menu-item-3302">
          <a href="http://dataunion.org/category/tech/ai" title="ai">
           人工智能
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-3303" id="menu-item-3303">
          <a href="http://dataunion.org/category/tech/analysis" title="analysis">
           数据分析
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-21920" id="menu-item-21920">
          <a href="http://dataunion.org/category/tech/dm">
           数据挖掘
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-3314" id="menu-item-3314">
          <a href="http://dataunion.org/category/tech/viz" title="viz">
           可视化
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category current-post-ancestor current-menu-parent current-post-parent menu-item-3305" id="menu-item-3305">
          <a href="http://dataunion.org/category/tech/devl" title="devl">
           编程语言
          </a>
         </li>
        </ul>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-has-children menu-item-20876" id="menu-item-20876">
        <a href="http://dataunion.org/category/industry">
         行业
        </a>
        <ul class="sub-menu">
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-16328" id="menu-item-16328">
          <a href="http://dataunion.org/category/industry/case" title="case">
           行业应用
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-2112" id="menu-item-2112">
          <a href="http://dataunion.org/category/industry/demo" title="demo">
           Demo展示
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-21562" id="menu-item-21562">
          <a href="http://dataunion.org/category/industry/news">
           行业资讯
          </a>
         </li>
        </ul>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-311" id="menu-item-311">
        <a href="http://dataunion.org/category/sources" title="sources">
         资源
        </a>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-20870" id="menu-item-20870">
        <a href="http://dataunion.org/category/books" title="book">
         图书
        </a>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-21363" id="menu-item-21363">
        <a href="http://dataunion.org/category/training">
         课程
        </a>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-has-children menu-item-21853" id="menu-item-21853">
        <a href="http://dataunion.org/category/jobs">
         职位
        </a>
        <ul class="sub-menu">
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-22050" id="menu-item-22050">
          <a href="http://dataunion.org/category/career">
           职业规划
          </a>
         </li>
        </ul>
       </li>
      </ul>
     </nav>
     <!--header-nav-->
    </div>
   </header>
   <!--header-web-->
   <div id="main">
    <div id="soutab">
     <form action="http://dataunion.org/" class="search" method="get">
     </form>
    </div>
    <div id="container">
     <nav id="mbx">
      当前位置：
      <a href="http://dataunion.org">
       首页
      </a>
      &gt;
      <a href="http://dataunion.org/category/tech">
       文章
      </a>
      &gt;
      <a href="http://dataunion.org/category/tech/devl">
       编程语言
      </a>
      &gt;  正文
     </nav>
     <!--mbx-->
     <article class="content">
      <header align="centre" class="contenttitle">
       <div class="mscc">
        <h1 class="mscctitle">
         <a href="http://dataunion.org/22887.html">
          Hadoop 平台问题解决思路集合
         </a>
        </h1>
        <address class="msccaddress ">
         <em>
          1,081 次阅读 -
         </em>
         <a href="http://dataunion.org/category/tech/devl" rel="category tag">
          编程语言
         </a>
        </address>
       </div>
      </header>
      <div class="content-text">
       <p>
        整理在工作中遇到的Hadoop平台遇到的报错信息，以及解决思路。
       </p>
       <h1 id="问题1：基于Yarn统一资源管理平台配置导致">
        问题1：基于Yarn统一资源管理平台配置导致
       </h1>
       <ul>
        <li>
         错误信息：
         <br/>
         <code>
          Application application_1458180019333_0002 failed 2 times due to AM Container
         </code>
         <br/>
         <code>
          for appattempt_1458180019333_0002_000002 exited with exitCode: -1000
         </code>
         <br/>
         <code>
          For more detailed output, check application tracking page:http://
         </code>
         <br/>
         <code>
          server-02:8088/proxy/application_1458180019333_0002/Then, click on links to
         </code>
         <br/>
         <code>
          logs of each attempt.Diagnostics: Not able to initialize user directories in
         </code>
         <br/>
         <code>
          any of the configured local directories for user hadoop
         </code>
         <br/>
         <code>
          Failing this attempt. Failing the application.
         </code>
        </li>
        <li>
         原因：Actually this is due to the permission issues on some of the yarn local directories. I started using LinuxContainerExecutor (in non secure mode with nonsecure-mode.local-user as kailash) and made corresponding changes. However due to some (unknown) reason NodeManager failed to clean local directories for users, and there still existed directories with previous user (in my case yarn).
         <br/>
         So to solve this, I first had to find the value of the property yarn.nodemanager.local-dirs (with Cloudera use search option to find this property for YARN service, otherwise look into yarn-site.xml in hadoop conf directory), and then delate the files/directories under usercache for all the node manager nodes. In my case, I used:
        </li>
       </ul>
       <blockquote>
        <p>
         rm -rf /yarn/nm/usercache/*
        </p>
       </blockquote>
       <ul>
        <li>
         问题分析：在配置impala基于yarn的完全自由管理，最后发现集群硬件严重不一致，llama项目让impala可以通过yarn申请资源，配置成功后，在执行小的复杂查询时没有任何问题，一旦数据量太大，会出现资源分配的问题，导致无法提供足够资源而报错，impalad mem_limit安装官网也做了限制了，并没有得到解决。最后取消配置，集群一切正常，再提交mapreducer,spark等基于yarn的计算引擎之时，发现各上面错误信息，一个
         <br/>
         任务都无法执行成功。排查问题步骤。
         <p>
         </p>
         <ul>
          <li>
           1、通过日志信息，确定大概方向。
          </li>
          <li>
           2、在开源社区各大国外论坛寻找解决方案，未果。
          </li>
          <li>
           3、排查源码，是什么导致输出次错误信息，发现是因为权限问题无法初始化。
          </li>
          <li>
           4、首先尝试找到相关yarn临时目录，备份目录后删除cache目录。重新提交任务初始化。
          </li>
          <li>
           5、整合解决问题时间大概在3个小时。就是一个权限问题。
           <ul>
            <li>
             配置基于yarn完全资源管理后权限：
             <br/>
             <figure class="highlight plain">
              <table>
               <tbody>
                <tr>
                 <td class="gutter">
                  <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
                  <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-57686a033d5d7017566329" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
                   <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
                    <span class="crayon-title">
                    </span>
                    <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
                     <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
                      <div class="crayon-button-icon">
                      </div>
                     </div>
                     <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
                      <div class="crayon-button-icon">
                      </div>
                     </div>
                     <div class="crayon-button crayon-wrap-button" title="切换自动换行">
                      <div class="crayon-button-icon">
                      </div>
                     </div>
                     <div class="crayon-button crayon-expand-button" title="点击展开代码">
                      <div class="crayon-button-icon">
                      </div>
                     </div>
                     <div class="crayon-button crayon-copy-button" title="复制代码">
                      <div class="crayon-button-icon">
                      </div>
                     </div>
                     <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
                      <div class="crayon-button-icon">
                      </div>
                     </div>
                    </div>
                   </div>
                   <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
                   </div>
                   <div class="crayon-plain-wrap">
                    <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
                     $ ls -l /data01/yarn/nm/usercache/
  total 1
  drwxr-s--- 4 nobody yarn 4096 Feb 24 00:19 hadoop
                    </textarea>
                   </div>
                   <div class="crayon-main" style="">
                    <table class="crayon-table">
                     <tbody>
                      <tr class="crayon-row">
                       <td class="crayon-nums " data-settings="show">
                        <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
                         <div class="crayon-num" data-line="crayon-57686a033d5d7017566329-1">
                          1
                         </div>
                         <div class="crayon-num crayon-striped-num" data-line="crayon-57686a033d5d7017566329-2">
                          2
                         </div>
                         <div class="crayon-num" data-line="crayon-57686a033d5d7017566329-3">
                          3
                         </div>
                        </div>
                       </td>
                       <td class="crayon-code">
                        <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
                         <div class="crayon-line" id="crayon-57686a033d5d7017566329-1">
                          <span class="crayon-sy">
                           $
                          </span>
                          <span class="crayon-h">
                          </span>
                          <span class="crayon-v">
                           ls
                          </span>
                          <span class="crayon-h">
                          </span>
                          <span class="crayon-o">
                           -
                          </span>
                          <span class="crayon-v">
                           l
                          </span>
                          <span class="crayon-h">
                          </span>
                          <span class="crayon-o">
                           /
                          </span>
                          <span class="crayon-v">
                           data01
                          </span>
                          <span class="crayon-o">
                           /
                          </span>
                          <span class="crayon-v">
                           yarn
                          </span>
                          <span class="crayon-o">
                           /
                          </span>
                          <span class="crayon-v">
                           nm
                          </span>
                          <span class="crayon-o">
                           /
                          </span>
                          <span class="crayon-v">
                           usercache
                          </span>
                          <span class="crayon-o">
                           /
                          </span>
                         </div>
                         <div class="crayon-line crayon-striped-line" id="crayon-57686a033d5d7017566329-2">
                          <span class="crayon-h">
                          </span>
                          <span class="crayon-i">
                           total
                          </span>
                          <span class="crayon-h">
                          </span>
                          <span class="crayon-cn">
                           1
                          </span>
                         </div>
                         <div class="crayon-line" id="crayon-57686a033d5d7017566329-3">
                          <span class="crayon-h">
                          </span>
                          <span class="crayon-v">
                           drwxr
                          </span>
                          <span class="crayon-o">
                           -
                          </span>
                          <span class="crayon-v">
                           s
                          </span>
                          <span class="crayon-o">
                           --
                          </span>
                          <span class="crayon-o">
                           -
                          </span>
                          <span class="crayon-h">
                          </span>
                          <span class="crayon-cn">
                           4
                          </span>
                          <span class="crayon-h">
                          </span>
                          <span class="crayon-e">
                           nobody
                          </span>
                          <span class="crayon-i">
                           yarn
                          </span>
                          <span class="crayon-h">
                          </span>
                          <span class="crayon-cn">
                           4096
                          </span>
                          <span class="crayon-h">
                          </span>
                          <span class="crayon-i">
                           Feb
                          </span>
                          <span class="crayon-h">
                          </span>
                          <span class="crayon-cn">
                           24
                          </span>
                          <span class="crayon-h">
                          </span>
                          <span class="crayon-cn">
                           00
                          </span>
                          <span class="crayon-o">
                           :
                          </span>
                          <span class="crayon-cn">
                           19
                          </span>
                          <span class="crayon-h">
                          </span>
                          <span class="crayon-v">
                           hadoop
                          </span>
                         </div>
                        </div>
                       </td>
                      </tr>
                     </tbody>
                    </table>
                   </div>
                  </div>
                  <!-- [Format Time: 0.0078 seconds] -->
                  <br/>
                 </td>
                 <td class="code">
                 </td>
                </tr>
               </tbody>
              </table>
             </figure>
            </li>
            <li>
             未配置基于yarn完全资源管理权限：
             <br/>
             <figure class="highlight plain">
              <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
              <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-57686a033d5e8290773872" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
               <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
                <span class="crayon-title">
                </span>
                <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
                 <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-wrap-button" title="切换自动换行">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-expand-button" title="点击展开代码">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-copy-button" title="复制代码">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <span class="crayon-language">
                  Java
                 </span>
                </div>
               </div>
               <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
               </div>
               <div class="crayon-plain-wrap">
                <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
                 $ ls -l /data01/yarn/nm/usercache/
total 1
drwxr-x--- 4 yarn yarn 4096 Mar 17 11:33 hadoop
                </textarea>
               </div>
               <div class="crayon-main" style="">
                <table class="crayon-table">
                 <tbody>
                  <tr class="crayon-row">
                   <td class="crayon-nums " data-settings="show">
                    <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
                     <div class="crayon-num" data-line="crayon-57686a033d5e8290773872-1">
                      1
                     </div>
                     <div class="crayon-num crayon-striped-num" data-line="crayon-57686a033d5e8290773872-2">
                      2
                     </div>
                     <div class="crayon-num" data-line="crayon-57686a033d5e8290773872-3">
                      3
                     </div>
                    </div>
                   </td>
                   <td class="crayon-code">
                    <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
                     <div class="crayon-line" id="crayon-57686a033d5e8290773872-1">
                      <span class="crayon-sy">
                       $
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-v">
                       ls
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-o">
                       -
                      </span>
                      <span class="crayon-v">
                       l
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-o">
                       /
                      </span>
                      <span class="crayon-v">
                       data01
                      </span>
                      <span class="crayon-o">
                       /
                      </span>
                      <span class="crayon-v">
                       yarn
                      </span>
                      <span class="crayon-o">
                       /
                      </span>
                      <span class="crayon-v">
                       nm
                      </span>
                      <span class="crayon-o">
                       /
                      </span>
                      <span class="crayon-v">
                       usercache
                      </span>
                      <span class="crayon-o">
                       /
                      </span>
                     </div>
                     <div class="crayon-line crayon-striped-line" id="crayon-57686a033d5e8290773872-2">
                      <span class="crayon-i">
                       total
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-cn">
                       1
                      </span>
                     </div>
                     <div class="crayon-line" id="crayon-57686a033d5e8290773872-3">
                      <span class="crayon-v">
                       drwxr
                      </span>
                      <span class="crayon-o">
                       -
                      </span>
                      <span class="crayon-v">
                       x
                      </span>
                      <span class="crayon-o">
                       --
                      </span>
                      <span class="crayon-o">
                       -
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-cn">
                       4
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       yarn
                      </span>
                      <span class="crayon-i">
                       yarn
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-cn">
                       4096
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-i">
                       Mar
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-cn">
                       17
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-cn">
                       11
                      </span>
                      <span class="crayon-o">
                       :
                      </span>
                      <span class="crayon-cn">
                       33
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-v">
                       hadoop
                      </span>
                     </div>
                    </div>
                   </td>
                  </tr>
                 </tbody>
                </table>
               </div>
              </div>
              <!-- [Format Time: 0.0019 seconds] -->
              <br/>
              <p>
              </p>
             </figure>
            </li>
           </ul>
          </li>
         </ul>
         <p>
          找到：yarn.nodemanager.local-dirs ，删除这个路径下面的
          <br/>
          usercache目录中的所有文件包括目录，即可解决。
         </p>
        </li>
       </ul>
       <h1 id="问题2：Container_xxx_is_running_beyond_physical_memory_limits">
        问题2：Container xxx is running beyond physical memory limits
       </h1>
       <ul>
        <li>
         日志：
         <br/>
         <figure class="highlight plain">
          <table>
           <tbody>
            <tr>
             <td class="gutter">
             </td>
             <td class="code">
              <p>
              </p>
              <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
              <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-57686a033d5ef976950875" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
               <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
                <span class="crayon-title">
                </span>
                <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
                 <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-wrap-button" title="切换自动换行">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-expand-button" title="点击展开代码">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-copy-button" title="复制代码">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <span class="crayon-language">
                  Java
                 </span>
                </div>
               </div>
               <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
               </div>
               <div class="crayon-plain-wrap">
                <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
                 Container [pid=134663,containerID=container_1430287094897_0049_02_067966] is running beyond physical memory limits. Current usage: 1.0 GB of 1 GB physical memory used; 1.5 GB of 10 GB virtual memory used. Killing container. Dump of the process-tree for
                </textarea>
               </div>
               <div class="crayon-main" style="">
                <table class="crayon-table">
                 <tbody>
                  <tr class="crayon-row">
                   <td class="crayon-nums " data-settings="show">
                    <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
                     <div class="crayon-num" data-line="crayon-57686a033d5ef976950875-1">
                      1
                     </div>
                    </div>
                   </td>
                   <td class="crayon-code">
                    <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
                     <div class="crayon-line" id="crayon-57686a033d5ef976950875-1">
                      <span class="crayon-i">
                       Container
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-sy">
                       [
                      </span>
                      <span class="crayon-v">
                       pid
                      </span>
                      <span class="crayon-o">
                       =
                      </span>
                      <span class="crayon-cn">
                       134663
                      </span>
                      <span class="crayon-sy">
                       ,
                      </span>
                      <span class="crayon-v">
                       containerID
                      </span>
                      <span class="crayon-o">
                       =
                      </span>
                      <span class="crayon-v">
                       container_1430287094897_0049_02_067966
                      </span>
                      <span class="crayon-sy">
                       ]
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-st">
                       is
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       running
                      </span>
                      <span class="crayon-e">
                       beyond
                      </span>
                      <span class="crayon-e">
                       physical
                      </span>
                      <span class="crayon-e">
                       memory
                      </span>
                      <span class="crayon-v">
                       limits
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       Current
                      </span>
                      <span class="crayon-v">
                       usage
                      </span>
                      <span class="crayon-o">
                       :
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-cn">
                       1.0
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       GB
                      </span>
                      <span class="crayon-i">
                       of
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-cn">
                       1
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       GB
                      </span>
                      <span class="crayon-e">
                       physical
                      </span>
                      <span class="crayon-e">
                       memory
                      </span>
                      <span class="crayon-v">
                       used
                      </span>
                      <span class="crayon-sy">
                       ;
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-cn">
                       1.5
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       GB
                      </span>
                      <span class="crayon-i">
                       of
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-cn">
                       10
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       GB
                      </span>
                      <span class="crayon-e">
                       virtual
                      </span>
                      <span class="crayon-e">
                       memory
                      </span>
                      <span class="crayon-v">
                       used
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       Killing
                      </span>
                      <span class="crayon-v">
                       container
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       Dump
                      </span>
                      <span class="crayon-e">
                       of
                      </span>
                      <span class="crayon-e">
                       the
                      </span>
                      <span class="crayon-v">
                       process
                      </span>
                      <span class="crayon-o">
                       -
                      </span>
                      <span class="crayon-e">
                       tree
                      </span>
                      <span class="crayon-st">
                       for
                      </span>
                     </div>
                    </div>
                   </td>
                  </tr>
                 </tbody>
                </table>
               </div>
              </div>
              <!-- [Format Time: 0.0113 seconds] -->
              <p>
              </p>
             </td>
            </tr>
           </tbody>
          </table>
         </figure>
        </li>
        <li>
         问题分析：
         <br/>
         从日志可以看出，container使用内存超过虚拟内存的限制，导致如上问题。默认2.1；
         <br/>
         NodeManager端设置，类似系统层面的overcommit问题,需要调节yarn.nodemanager.vmem-pmem-ratio相关参数，在yarn-site.xml修改:
         <br/>
         <figure class="highlight plain">
          <table>
           <tbody>
            <tr>
             <td class="gutter">
             </td>
             <td class="code">
              <p>
              </p>
              <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
              <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-57686a033d5f6971362565" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
               <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
                <span class="crayon-title">
                </span>
                <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
                 <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-wrap-button" title="切换自动换行">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-expand-button" title="点击展开代码">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-copy-button" title="复制代码">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <span class="crayon-language">
                  Java
                 </span>
                </div>
               </div>
               <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
               </div>
               <div class="crayon-plain-wrap">
                <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
                 &lt;property&gt;
 &lt;name&gt;yarn.nodemanager.vmem-pmem-ratio&lt;/name&gt;
     &lt;value&gt;10&lt;/value&gt;
 &lt;/property&gt;
 –或者yarn.nodemanager.vmem-check-enabled，false掉 
 &lt;property&gt;
     &lt;name&gt;yarn.nodemanager.vmem-check-enabled&lt;/name&gt;
     &lt;value&gt;false&lt;/value&gt;
 &lt;/property&gt;
                </textarea>
               </div>
               <div class="crayon-main" style="">
                <table class="crayon-table">
                 <tbody>
                  <tr class="crayon-row">
                   <td class="crayon-nums " data-settings="show">
                    <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
                     <div class="crayon-num" data-line="crayon-57686a033d5f6971362565-1">
                      1
                     </div>
                     <div class="crayon-num crayon-striped-num" data-line="crayon-57686a033d5f6971362565-2">
                      2
                     </div>
                     <div class="crayon-num" data-line="crayon-57686a033d5f6971362565-3">
                      3
                     </div>
                     <div class="crayon-num crayon-striped-num" data-line="crayon-57686a033d5f6971362565-4">
                      4
                     </div>
                     <div class="crayon-num" data-line="crayon-57686a033d5f6971362565-5">
                      5
                     </div>
                     <div class="crayon-num crayon-striped-num" data-line="crayon-57686a033d5f6971362565-6">
                      6
                     </div>
                     <div class="crayon-num" data-line="crayon-57686a033d5f6971362565-7">
                      7
                     </div>
                     <div class="crayon-num crayon-striped-num" data-line="crayon-57686a033d5f6971362565-8">
                      8
                     </div>
                     <div class="crayon-num" data-line="crayon-57686a033d5f6971362565-9">
                      9
                     </div>
                    </div>
                   </td>
                   <td class="crayon-code">
                    <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
                     <div class="crayon-line" id="crayon-57686a033d5f6971362565-1">
                      <span class="crayon-e ">
                       &lt;property&gt;
                      </span>
                     </div>
                     <div class="crayon-line crayon-striped-line" id="crayon-57686a033d5f6971362565-2">
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e ">
                       &lt;name&gt;
                      </span>
                      <span class="crayon-v">
                       yarn
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       nodemanager
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       vmem
                      </span>
                      <span class="crayon-o">
                       -
                      </span>
                      <span class="crayon-v">
                       pmem
                      </span>
                      <span class="crayon-o">
                       -
                      </span>
                      <span class="crayon-v">
                       ratio
                      </span>
                      <span class="crayon-o">
                       &lt;
                      </span>
                      <span class="crayon-o">
                       /
                      </span>
                      <span class="crayon-v">
                       name
                      </span>
                      <span class="crayon-o">
                       &gt;
                      </span>
                     </div>
                     <div class="crayon-line" id="crayon-57686a033d5f6971362565-3">
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e ">
                       &lt;value&gt;
                      </span>
                      <span class="crayon-cn">
                       10
                      </span>
                      <span class="crayon-o">
                       &lt;
                      </span>
                      <span class="crayon-o">
                       /
                      </span>
                      <span class="crayon-v">
                       value
                      </span>
                      <span class="crayon-o">
                       &gt;
                      </span>
                     </div>
                     <div class="crayon-line crayon-striped-line" id="crayon-57686a033d5f6971362565-4">
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-o">
                       &lt;
                      </span>
                      <span class="crayon-o">
                       /
                      </span>
                      <span class="crayon-m">
                       property
                      </span>
                      <span class="crayon-o">
                       &gt;
                      </span>
                     </div>
                     <div class="crayon-line" id="crayon-57686a033d5f6971362565-5">
                      <span class="crayon-h">
                      </span>
                      –或者
                      <span class="crayon-v">
                       yarn
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       nodemanager
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       vmem
                      </span>
                      <span class="crayon-o">
                       -
                      </span>
                      <span class="crayon-v">
                       check
                      </span>
                      <span class="crayon-o">
                       -
                      </span>
                      <span class="crayon-i">
                       enabled
                      </span>
                      ，
                      <span class="crayon-t">
                       false
                      </span>
                      掉
                      <span class="crayon-h">
                      </span>
                     </div>
                     <div class="crayon-line crayon-striped-line" id="crayon-57686a033d5f6971362565-6">
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e ">
                       &lt;property&gt;
                      </span>
                     </div>
                     <div class="crayon-line" id="crayon-57686a033d5f6971362565-7">
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e ">
                       &lt;name&gt;
                      </span>
                      <span class="crayon-v">
                       yarn
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       nodemanager
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       vmem
                      </span>
                      <span class="crayon-o">
                       -
                      </span>
                      <span class="crayon-v">
                       check
                      </span>
                      <span class="crayon-o">
                       -
                      </span>
                      <span class="crayon-v">
                       enabled
                      </span>
                      <span class="crayon-o">
                       &lt;
                      </span>
                      <span class="crayon-o">
                       /
                      </span>
                      <span class="crayon-v">
                       name
                      </span>
                      <span class="crayon-o">
                       &gt;
                      </span>
                     </div>
                     <div class="crayon-line crayon-striped-line" id="crayon-57686a033d5f6971362565-8">
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e ">
                       &lt;value&gt;
                      </span>
                      <span class="crayon-t">
                       false
                      </span>
                      <span class="crayon-o">
                       &lt;
                      </span>
                      <span class="crayon-o">
                       /
                      </span>
                      <span class="crayon-v">
                       value
                      </span>
                      <span class="crayon-o">
                       &gt;
                      </span>
                     </div>
                     <div class="crayon-line" id="crayon-57686a033d5f6971362565-9">
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-o">
                       &lt;
                      </span>
                      <span class="crayon-o">
                       /
                      </span>
                      <span class="crayon-m">
                       property
                      </span>
                      <span class="crayon-o">
                       &gt;
                      </span>
                     </div>
                    </div>
                   </td>
                  </tr>
                 </tbody>
                </table>
               </div>
              </div>
              <!-- [Format Time: 0.0119 seconds] -->
              <p>
              </p>
             </td>
            </tr>
           </tbody>
          </table>
         </figure>
        </li>
       </ul>
       <h1 id="问题3：jvm系常见java_heap_space">
        问题3：jvm系常见java heap space
       </h1>
       <ul>
        <li>
         日志
         <br/>
         <figure class="highlight plain">
          <table>
           <tbody>
            <tr>
             <td class="gutter">
              <p>
              </p>
              <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
              <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-57686a033d5fc280708481" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
               <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
                <span class="crayon-title">
                </span>
                <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
                 <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-wrap-button" title="切换自动换行">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-expand-button" title="点击展开代码">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-copy-button" title="复制代码">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                </div>
               </div>
               <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
               </div>
               <div class="crayon-plain-wrap">
                <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
                 xxx java heap space/ java heap space xxx 各种oom信息
                </textarea>
               </div>
               <div class="crayon-main" style="">
                <table class="crayon-table">
                 <tbody>
                  <tr class="crayon-row">
                   <td class="crayon-nums " data-settings="show">
                    <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
                     <div class="crayon-num" data-line="crayon-57686a033d5fc280708481-1">
                      1
                     </div>
                    </div>
                   </td>
                   <td class="crayon-code">
                    <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
                     <div class="crayon-line" id="crayon-57686a033d5fc280708481-1">
                      <span class="crayon-e">
                       xxx
                      </span>
                      <span class="crayon-e">
                       java
                      </span>
                      <span class="crayon-e">
                       heap
                      </span>
                      <span class="crayon-v">
                       space
                      </span>
                      <span class="crayon-o">
                       /
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       java
                      </span>
                      <span class="crayon-e">
                       heap
                      </span>
                      <span class="crayon-e">
                       space
                      </span>
                      <span class="crayon-i">
                       xxx
                      </span>
                      <span class="crayon-h">
                      </span>
                      各种
                      <span class="crayon-i">
                       oom
                      </span>
                      信息
                     </div>
                    </div>
                   </td>
                  </tr>
                 </tbody>
                </table>
               </div>
              </div>
              <!-- [Format Time: 0.0066 seconds] -->
              <p>
              </p>
             </td>
             <td class="code">
             </td>
            </tr>
           </tbody>
          </table>
         </figure>
        </li>
        <li>
         问题分析
         <br/>
         <figure class="highlight plain">
          <table>
           <tbody>
            <tr>
             <td class="gutter">
              <p>
              </p>
              <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
              <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-57686a033d602354950473" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
               <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
                <span class="crayon-title">
                </span>
                <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
                 <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-wrap-button" title="切换自动换行">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-expand-button" title="点击展开代码">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-copy-button" title="复制代码">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                </div>
               </div>
               <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
               </div>
               <div class="crayon-plain-wrap">
                <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
                 通过日志定位问题，一般调节参数思路，相关mem,heap设置参数：
  -内存：mapreduce.map.memory.mb
  –Heap Size：-Xmx在mapreduce.map.java.opts做相同调整
  –内存：mapreduce.reduce.memory.mb
  –Heap Size：-Xmx在mapreduce.reduce.java.opts做相同调整
                </textarea>
               </div>
               <div class="crayon-main" style="">
                <table class="crayon-table">
                 <tbody>
                  <tr class="crayon-row">
                   <td class="crayon-nums " data-settings="show">
                    <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
                     <div class="crayon-num" data-line="crayon-57686a033d602354950473-1">
                      1
                     </div>
                     <div class="crayon-num crayon-striped-num" data-line="crayon-57686a033d602354950473-2">
                      2
                     </div>
                     <div class="crayon-num" data-line="crayon-57686a033d602354950473-3">
                      3
                     </div>
                     <div class="crayon-num crayon-striped-num" data-line="crayon-57686a033d602354950473-4">
                      4
                     </div>
                     <div class="crayon-num" data-line="crayon-57686a033d602354950473-5">
                      5
                     </div>
                    </div>
                   </td>
                   <td class="crayon-code">
                    <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
                     <div class="crayon-line" id="crayon-57686a033d602354950473-1">
                      通过日志定位问题，一般调节参数思路，相关
                      <span class="crayon-v">
                       mem
                      </span>
                      <span class="crayon-sy">
                       ,
                      </span>
                      <span class="crayon-i">
                       heap
                      </span>
                      设置参数：
                     </div>
                     <div class="crayon-line crayon-striped-line" id="crayon-57686a033d602354950473-2">
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-o">
                       -
                      </span>
                      内存：
                      <span class="crayon-v">
                       mapreduce
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       map
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       memory
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-i">
                       mb
                      </span>
                     </div>
                     <div class="crayon-line" id="crayon-57686a033d602354950473-3">
                      <span class="crayon-h">
                      </span>
                      –
                      <span class="crayon-e">
                       Heap
                      </span>
                      <span class="crayon-i">
                       Size
                      </span>
                      ：
                      <span class="crayon-o">
                       -
                      </span>
                      <span class="crayon-i">
                       Xmx
                      </span>
                      在
                      <span class="crayon-v">
                       mapreduce
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       map
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       java
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-i">
                       opts
                      </span>
                      做相同调整
                     </div>
                     <div class="crayon-line crayon-striped-line" id="crayon-57686a033d602354950473-4">
                      <span class="crayon-h">
                      </span>
                      –内存：
                      <span class="crayon-v">
                       mapreduce
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       reduce
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       memory
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-i">
                       mb
                      </span>
                     </div>
                     <div class="crayon-line" id="crayon-57686a033d602354950473-5">
                      <span class="crayon-h">
                      </span>
                      –
                      <span class="crayon-e">
                       Heap
                      </span>
                      <span class="crayon-i">
                       Size
                      </span>
                      ：
                      <span class="crayon-o">
                       -
                      </span>
                      <span class="crayon-i">
                       Xmx
                      </span>
                      在
                      <span class="crayon-v">
                       mapreduce
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       reduce
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       java
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-i">
                       opts
                      </span>
                      做相同调整
                     </div>
                    </div>
                   </td>
                  </tr>
                 </tbody>
                </table>
               </div>
              </div>
              <!-- [Format Time: 0.0084 seconds] -->
              <p>
              </p>
             </td>
             <td class="code">
             </td>
            </tr>
           </tbody>
          </table>
         </figure>
        </li>
        <li>
         yarn资源管理&amp;&amp;优化参考如下文章
         <ul>
          <li>
           <a href="http://www.itweet.cn/2015/07/24/yarn-resources-manager-allocation/">
            yarn-resources-manager-allocation
           </a>
          </li>
         </ul>
        </li>
       </ul>
       <h1 id="问题4:_llama_on_yarn使用问题">
        问题4: llama on yarn使用问题
       </h1>
       <ul>
        <li>
         日志
         <br/>
         <figure class="highlight plain">
          <table>
           <tbody>
            <tr>
             <td class="gutter">
             </td>
             <td class="code">
              <p>
              </p>
              <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
              <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-57686a033d609827636521" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
               <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
                <span class="crayon-title">
                </span>
                <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
                 <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-wrap-button" title="切换自动换行">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-expand-button" title="点击展开代码">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-copy-button" title="复制代码">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <span class="crayon-language">
                  Java
                 </span>
                </div>
               </div>
               <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
               </div>
               <div class="crayon-plain-wrap">
                <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
                 ERROR:  com.cloudera.llama.util.LlamaException: RESERVATION_ASKING_MORE_MB - Reservation '44590e9da89d8e3:a5ba70d7dcb599a3', expansion 'null' is asking for more memory in mb '129008' than capacity '76800' on node 'bigdata-server-04'., com.cloudera.llama.util.LlamaException: RESERVATION_ASKING_MORE_MB - Reservation '44590e9da89d8e3:a5ba70d7dcb599a3', expansion 'null' is asking for more memory in mb '129008' than capacity '76800' on node 'server-04'.,        at com.cloudera.llama.am.impl.ExpansionReservationsLlamaAM.checkAndUpdateCapacity(ExpansionReservationsLlamaAM.java:170),       at com.cloudera.llama.am.impl.ExpansionReservationsLlamaAM.reserve(ExpansionReservationsLlamaAM.java:129),     at com.cloudera.llama.am.impl.APIContractLlamaAM.reserve(APIContractLlamaAM.java:144),         at com.cloudera.llama.am.LlamaAMServiceImpl.Reserve(LlamaAMServiceImpl.java:132),       at com.cloudera.llama.am.MetricLlamaAMService.Reserve(MetricLlamaAMService.java:140),  at com.cloudera.llama.thrift.LlamaAMService$Processor$Reserve.getResult(LlamaAMService.java:512),      at com.cloudera.llama.thrift.LlamaAMService$Processor$Reserve.getResult(LlamaAMService.java:497),       at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39),         at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39),    at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:206),   at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145),    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615),     at java.lang.Thread.run(Thread.java:745)
WARNING: The following tables are missing relevant table and/or column statistics.
bigdata.terminal_all
                </textarea>
               </div>
               <div class="crayon-main" style="">
                <table class="crayon-table">
                 <tbody>
                  <tr class="crayon-row">
                   <td class="crayon-nums " data-settings="show">
                    <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
                     <div class="crayon-num" data-line="crayon-57686a033d609827636521-1">
                      1
                     </div>
                     <div class="crayon-num crayon-striped-num" data-line="crayon-57686a033d609827636521-2">
                      2
                     </div>
                     <div class="crayon-num" data-line="crayon-57686a033d609827636521-3">
                      3
                     </div>
                    </div>
                   </td>
                   <td class="crayon-code">
                    <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
                     <div class="crayon-line" id="crayon-57686a033d609827636521-1">
                      <span class="crayon-v">
                       ERROR
                      </span>
                      <span class="crayon-o">
                       :
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-v">
                       com
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       cloudera
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       llama
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       util
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       LlamaException
                      </span>
                      <span class="crayon-o">
                       :
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-v">
                       RESERVATION_ASKING_MORE_MB
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-o">
                       -
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-i">
                       Reservation
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-s">
                       '44590e9da89d8e3:a5ba70d7dcb599a3'
                      </span>
                      <span class="crayon-sy">
                       ,
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-i">
                       expansion
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-s">
                       'null'
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-st">
                       is
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       asking
                      </span>
                      <span class="crayon-st">
                       for
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       more
                      </span>
                      <span class="crayon-e">
                       memory
                      </span>
                      <span class="crayon-st">
                       in
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-i">
                       mb
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-s">
                       '129008'
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       than
                      </span>
                      <span class="crayon-i">
                       capacity
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-s">
                       '76800'
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       on
                      </span>
                      <span class="crayon-i">
                       node
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-s">
                       'bigdata-server-04'
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-sy">
                       ,
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-v">
                       com
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       cloudera
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       llama
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       util
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       LlamaException
                      </span>
                      <span class="crayon-o">
                       :
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-v">
                       RESERVATION_ASKING_MORE_MB
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-o">
                       -
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-i">
                       Reservation
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-s">
                       '44590e9da89d8e3:a5ba70d7dcb599a3'
                      </span>
                      <span class="crayon-sy">
                       ,
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-i">
                       expansion
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-s">
                       'null'
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-st">
                       is
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       asking
                      </span>
                      <span class="crayon-st">
                       for
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       more
                      </span>
                      <span class="crayon-e">
                       memory
                      </span>
                      <span class="crayon-st">
                       in
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-i">
                       mb
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-s">
                       '129008'
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       than
                      </span>
                      <span class="crayon-i">
                       capacity
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-s">
                       '76800'
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       on
                      </span>
                      <span class="crayon-i">
                       node
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-s">
                       'server-04'
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-sy">
                       ,
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       at
                      </span>
                      <span class="crayon-v">
                       com
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       cloudera
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       llama
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       am
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       impl
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       ExpansionReservationsLlamaAM
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-e">
                       checkAndUpdateCapacity
                      </span>
                      <span class="crayon-sy">
                       (
                      </span>
                      <span class="crayon-v">
                       ExpansionReservationsLlamaAM
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       java
                      </span>
                      <span class="crayon-o">
                       :
                      </span>
                      <span class="crayon-cn">
                       170
                      </span>
                      <span class="crayon-sy">
                       )
                      </span>
                      <span class="crayon-sy">
                       ,
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       at
                      </span>
                      <span class="crayon-v">
                       com
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       cloudera
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       llama
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       am
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       impl
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       ExpansionReservationsLlamaAM
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-e">
                       reserve
                      </span>
                      <span class="crayon-sy">
                       (
                      </span>
                      <span class="crayon-v">
                       ExpansionReservationsLlamaAM
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       java
                      </span>
                      <span class="crayon-o">
                       :
                      </span>
                      <span class="crayon-cn">
                       129
                      </span>
                      <span class="crayon-sy">
                       )
                      </span>
                      <span class="crayon-sy">
                       ,
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       at
                      </span>
                      <span class="crayon-v">
                       com
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       cloudera
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       llama
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       am
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       impl
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       APIContractLlamaAM
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-e">
                       reserve
                      </span>
                      <span class="crayon-sy">
                       (
                      </span>
                      <span class="crayon-v">
                       APIContractLlamaAM
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       java
                      </span>
                      <span class="crayon-o">
                       :
                      </span>
                      <span class="crayon-cn">
                       144
                      </span>
                      <span class="crayon-sy">
                       )
                      </span>
                      <span class="crayon-sy">
                       ,
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       at
                      </span>
                      <span class="crayon-v">
                       com
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       cloudera
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       llama
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       am
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       LlamaAMServiceImpl
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-e">
                       Reserve
                      </span>
                      <span class="crayon-sy">
                       (
                      </span>
                      <span class="crayon-v">
                       LlamaAMServiceImpl
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       java
                      </span>
                      <span class="crayon-o">
                       :
                      </span>
                      <span class="crayon-cn">
                       132
                      </span>
                      <span class="crayon-sy">
                       )
                      </span>
                      <span class="crayon-sy">
                       ,
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       at
                      </span>
                      <span class="crayon-v">
                       com
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       cloudera
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       llama
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       am
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       MetricLlamaAMService
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-e">
                       Reserve
                      </span>
                      <span class="crayon-sy">
                       (
                      </span>
                      <span class="crayon-v">
                       MetricLlamaAMService
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       java
                      </span>
                      <span class="crayon-o">
                       :
                      </span>
                      <span class="crayon-cn">
                       140
                      </span>
                      <span class="crayon-sy">
                       )
                      </span>
                      <span class="crayon-sy">
                       ,
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       at
                      </span>
                      <span class="crayon-v">
                       com
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       cloudera
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       llama
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       thrift
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       LlamaAMService
                      </span>
                      <span class="crayon-sy">
                       $
                      </span>
                      <span class="crayon-v">
                       Processor
                      </span>
                      <span class="crayon-sy">
                       $
                      </span>
                      <span class="crayon-v">
                       Reserve
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-e">
                       getResult
                      </span>
                      <span class="crayon-sy">
                       (
                      </span>
                      <span class="crayon-v">
                       LlamaAMService
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       java
                      </span>
                      <span class="crayon-o">
                       :
                      </span>
                      <span class="crayon-cn">
                       512
                      </span>
                      <span class="crayon-sy">
                       )
                      </span>
                      <span class="crayon-sy">
                       ,
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       at
                      </span>
                      <span class="crayon-v">
                       com
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       cloudera
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       llama
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       thrift
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       LlamaAMService
                      </span>
                      <span class="crayon-sy">
                       $
                      </span>
                      <span class="crayon-v">
                       Processor
                      </span>
                      <span class="crayon-sy">
                       $
                      </span>
                      <span class="crayon-v">
                       Reserve
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-e">
                       getResult
                      </span>
                      <span class="crayon-sy">
                       (
                      </span>
                      <span class="crayon-v">
                       LlamaAMService
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       java
                      </span>
                      <span class="crayon-o">
                       :
                      </span>
                      <span class="crayon-cn">
                       497
                      </span>
                      <span class="crayon-sy">
                       )
                      </span>
                      <span class="crayon-sy">
                       ,
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       at
                      </span>
                      <span class="crayon-v">
                       org
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       apache
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       thrift
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       ProcessFunction
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-e">
                       process
                      </span>
                      <span class="crayon-sy">
                       (
                      </span>
                      <span class="crayon-v">
                       ProcessFunction
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       java
                      </span>
                      <span class="crayon-o">
                       :
                      </span>
                      <span class="crayon-cn">
                       39
                      </span>
                      <span class="crayon-sy">
                       )
                      </span>
                      <span class="crayon-sy">
                       ,
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       at
                      </span>
                      <span class="crayon-v">
                       org
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       apache
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       thrift
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       TBaseProcessor
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-e">
                       process
                      </span>
                      <span class="crayon-sy">
                       (
                      </span>
                      <span class="crayon-v">
                       TBaseProcessor
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       java
                      </span>
                      <span class="crayon-o">
                       :
                      </span>
                      <span class="crayon-cn">
                       39
                      </span>
                      <span class="crayon-sy">
                       )
                      </span>
                      <span class="crayon-sy">
                       ,
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       at
                      </span>
                      <span class="crayon-v">
                       org
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       apache
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       thrift
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       server
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       TThreadPoolServer
                      </span>
                      <span class="crayon-sy">
                       $
                      </span>
                      <span class="crayon-v">
                       WorkerProcess
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-e">
                       run
                      </span>
                      <span class="crayon-sy">
                       (
                      </span>
                      <span class="crayon-v">
                       TThreadPoolServer
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       java
                      </span>
                      <span class="crayon-o">
                       :
                      </span>
                      <span class="crayon-cn">
                       206
                      </span>
                      <span class="crayon-sy">
                       )
                      </span>
                      <span class="crayon-sy">
                       ,
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       at
                      </span>
                      <span class="crayon-v">
                       java
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       util
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       concurrent
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       ThreadPoolExecutor
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-e">
                       runWorker
                      </span>
                      <span class="crayon-sy">
                       (
                      </span>
                      <span class="crayon-v">
                       ThreadPoolExecutor
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       java
                      </span>
                      <span class="crayon-o">
                       :
                      </span>
                      <span class="crayon-cn">
                       1145
                      </span>
                      <span class="crayon-sy">
                       )
                      </span>
                      <span class="crayon-sy">
                       ,
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       at
                      </span>
                      <span class="crayon-v">
                       java
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       util
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       concurrent
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       ThreadPoolExecutor
                      </span>
                      <span class="crayon-sy">
                       $
                      </span>
                      <span class="crayon-v">
                       Worker
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-e">
                       run
                      </span>
                      <span class="crayon-sy">
                       (
                      </span>
                      <span class="crayon-v">
                       ThreadPoolExecutor
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       java
                      </span>
                      <span class="crayon-o">
                       :
                      </span>
                      <span class="crayon-cn">
                       615
                      </span>
                      <span class="crayon-sy">
                       )
                      </span>
                      <span class="crayon-sy">
                       ,
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       at
                      </span>
                      <span class="crayon-v">
                       java
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       lang
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       Thread
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-e">
                       run
                      </span>
                      <span class="crayon-sy">
                       (
                      </span>
                      <span class="crayon-v">
                       Thread
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       java
                      </span>
                      <span class="crayon-o">
                       :
                      </span>
                      <span class="crayon-cn">
                       745
                      </span>
                      <span class="crayon-sy">
                       )
                      </span>
                     </div>
                     <div class="crayon-line crayon-striped-line" id="crayon-57686a033d609827636521-2">
                      <span class="crayon-v">
                       WARNING
                      </span>
                      <span class="crayon-o">
                       :
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       The
                      </span>
                      <span class="crayon-e">
                       following
                      </span>
                      <span class="crayon-e">
                       tables
                      </span>
                      <span class="crayon-e">
                       are
                      </span>
                      <span class="crayon-e">
                       missing
                      </span>
                      <span class="crayon-e">
                       relevant
                      </span>
                      <span class="crayon-e">
                       table
                      </span>
                      <span class="crayon-st">
                       and
                      </span>
                      <span class="crayon-o">
                       /
                      </span>
                      <span class="crayon-st">
                       or
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       column
                      </span>
                      <span class="crayon-v">
                       statistics
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                     </div>
                     <div class="crayon-line" id="crayon-57686a033d609827636521-3">
                      <span class="crayon-v">
                       bigdata
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       terminal_all
                      </span>
                     </div>
                    </div>
                   </td>
                  </tr>
                 </tbody>
                </table>
               </div>
              </div>
              <!-- [Format Time: 0.0280 seconds] -->
              <p>
              </p>
             </td>
            </tr>
           </tbody>
          </table>
         </figure>
        </li>
        <li>
         分析问题
         <br/>
         <figure class="highlight plain">
          <table>
           <tbody>
            <tr>
             <td class="gutter">
             </td>
             <td class="code">
              <p>
              </p>
              <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
              <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-57686a033d610110767197" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
               <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
                <span class="crayon-title">
                </span>
                <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
                 <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-wrap-button" title="切换自动换行">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-expand-button" title="点击展开代码">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-copy-button" title="复制代码">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                </div>
               </div>
               <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
               </div>
               <div class="crayon-plain-wrap">
                <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
                 &lt;span class="line"&gt;根据错误日志，去cloudera官网blog各种翻文档，查看相关内存配置参数和限制，集群硬件mem严重不一致，在资源评估分配的时候，impalad分别在这些不同配置mem的节点，
根据官网说法mem_limit也做了限制，依然无法解决。impalad申请内存是非常夸张的方式，如下参数mem调整。

==&amp;gt; Impala Daemon 内存限制，mem_limit，impalad内存设置导致问题：导致的问题如下，需要限制impalad
内存使用，注意如果一个集群有多种不同硬件配置
yarn.nodemanager.resource.memory-mb这个参数注意限制。

单表数据量太大，虽然做了各种限制，但是依然会报错，未解决，还不够成熟和yarn集成，取消llama配置，独立部署模式就可以了，一模一样的SQL可以执行成功，我无言以对。&lt;/span&gt;
                </textarea>
               </div>
               <div class="crayon-main" style="">
                <table class="crayon-table">
                 <tbody>
                  <tr class="crayon-row">
                   <td class="crayon-nums " data-settings="show">
                    <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
                     <div class="crayon-num" data-line="crayon-57686a033d610110767197-1">
                      1
                     </div>
                     <div class="crayon-num crayon-striped-num" data-line="crayon-57686a033d610110767197-2">
                      2
                     </div>
                     <div class="crayon-num" data-line="crayon-57686a033d610110767197-3">
                      3
                     </div>
                     <div class="crayon-num crayon-striped-num" data-line="crayon-57686a033d610110767197-4">
                      4
                     </div>
                     <div class="crayon-num" data-line="crayon-57686a033d610110767197-5">
                      5
                     </div>
                     <div class="crayon-num crayon-striped-num" data-line="crayon-57686a033d610110767197-6">
                      6
                     </div>
                     <div class="crayon-num" data-line="crayon-57686a033d610110767197-7">
                      7
                     </div>
                     <div class="crayon-num crayon-striped-num" data-line="crayon-57686a033d610110767197-8">
                      8
                     </div>
                    </div>
                   </td>
                   <td class="crayon-code">
                    <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
                     <div class="crayon-line" id="crayon-57686a033d610110767197-1">
                      <span class="crayon-o">
                       &lt;
                      </span>
                      <span class="crayon-e">
                       span
                      </span>
                      <span class="crayon-t">
                       class
                      </span>
                      <span class="crayon-o">
                       =
                      </span>
                      <span class="crayon-s">
                       "line"
                      </span>
                      <span class="crayon-o">
                       &gt;
                      </span>
                      根据错误日志，去
                      <span class="crayon-i">
                       cloudera
                      </span>
                      官网
                      <span class="crayon-i">
                       blog
                      </span>
                      各种翻文档，查看相关内存配置参数和限制，集群硬件
                      <span class="crayon-i">
                       mem
                      </span>
                      严重不一致，在资源评估分配的时候，
                      <span class="crayon-i">
                       impalad
                      </span>
                      分别在这些不同配置
                      <span class="crayon-i">
                       mem
                      </span>
                      的节点，
                     </div>
                     <div class="crayon-line crayon-striped-line" id="crayon-57686a033d610110767197-2">
                      根据官网说法
                      <span class="crayon-v">
                       mem
                      </span>
                      <span class="crayon-sy">
                       _
                      </span>
                      limit也做了限制，依然无法解决。
                      <span class="crayon-i">
                       impalad
                      </span>
                      申请内存是非常夸张的方式，如下参数
                      <span class="crayon-i">
                       mem
                      </span>
                      调整。
                     </div>
                     <div class="crayon-line" id="crayon-57686a033d610110767197-3">
                     </div>
                     <div class="crayon-line crayon-striped-line" id="crayon-57686a033d610110767197-4">
                      <span class="crayon-o">
                       ==
                      </span>
                      <span class="crayon-o">
                       &amp;
                      </span>
                      <span class="crayon-v">
                       gt
                      </span>
                      <span class="crayon-sy">
                       ;
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       Impala
                      </span>
                      <span class="crayon-i">
                       Daemon
                      </span>
                      <span class="crayon-h">
                      </span>
                      内存限制，
                      <span class="crayon-v">
                       mem
                      </span>
                      <span class="crayon-sy">
                       _
                      </span>
                      limit，
                      <span class="crayon-i">
                       impalad
                      </span>
                      内存设置导致问题：导致的问题如下，需要限制
                      <span class="crayon-i">
                       impalad
                      </span>
                     </div>
                     <div class="crayon-line" id="crayon-57686a033d610110767197-5">
                      内存使用，注意如果一个集群有多种不同硬件配置
                     </div>
                     <div class="crayon-line crayon-striped-line" id="crayon-57686a033d610110767197-6">
                      <span class="crayon-v">
                       yarn
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       nodemanager
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       resource
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       memory
                      </span>
                      <span class="crayon-o">
                       -
                      </span>
                      <span class="crayon-i">
                       mb
                      </span>
                      这个参数注意限制。
                     </div>
                     <div class="crayon-line" id="crayon-57686a033d610110767197-7">
                     </div>
                     <div class="crayon-line crayon-striped-line" id="crayon-57686a033d610110767197-8">
                      单表数据量太大，虽然做了各种限制，但是依然会报错，未解决，还不够成熟和
                      <span class="crayon-i">
                       yarn
                      </span>
                      集成，取消
                      <span class="crayon-i">
                       llama
                      </span>
                      配置，独立部署模式就可以了，一模一样的
                      <span class="crayon-i">
                       SQL
                      </span>
                      可以执行成功，我无言以对。
                      <span class="crayon-o">
                       &lt;
                      </span>
                      <span class="crayon-o">
                       /
                      </span>
                      <span class="crayon-v">
                       span
                      </span>
                      <span class="crayon-o">
                       &gt;
                      </span>
                     </div>
                    </div>
                   </td>
                  </tr>
                 </tbody>
                </table>
               </div>
              </div>
              <!-- [Format Time: 0.0110 seconds] -->
              <p>
              </p>
             </td>
            </tr>
           </tbody>
          </table>
         </figure>
        </li>
        <li>
         问题小结问题原因：由于集群中，上中下有三种配置服务器，内存在65-128g，128-256g都有，而hdfs，impala,hbase,yarn,sparkstreaming，prestodb都分布在这些节点上面，impalad是必须和datanode在一个节点，这就涉及到资源分配问题。通过简单对集群角色3组，不同组角色在不同硬件资源，mem,cpu,磁盘有不同的三套配置。llama项目使得impala资源分配交给Yarn。在这样资源严重不一致情况下。llama评估资源在某些节点实际资源就不和大环境统一，从而出现问题。解决问题方案：尽快引用hadoop 2.7+版本提高的基于标签调度策略，可以给集群不同
         <br/>
         nodemanager节点打标签，提交任务的时候可以指定什么类型应用提交到那些节点执行，
         <br/>
         从而在同一个集群中实现异构集群的思路。YARN某些资源池下面拥有某些高性能高配置
         <br/>
         服务器，某些低性能低配置在一个资源池，某些不上不下配置的服务器，同时拥有高性能标签，低性能标签。调度的时候，会同时存在不同的技术引擎的任务；比如：内存计算代表:spark,tez,impala,drill,presto，磁盘计算：mr,hive,pig,mahout，流计算：sparkstreaming,storm。不同任务指定到不同的资源池。达到基于YARN的统一资源管理和调度。
        </li>
       </ul>
       <h1 id="问题5：升级Namenode_HA出现Hive_metadata信息访问hdfs问题">
        问题5：升级Namenode HA出现Hive metadata信息访问hdfs问题
       </h1>
       <ul>
        <li>
         日志：
         <br/>
         <figure class="highlight plain">
          <table>
           <tbody>
            <tr>
             <td class="gutter">
              <p>
              </p>
              <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
              <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-57686a033d618513631619" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
               <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
                <span class="crayon-title">
                </span>
                <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
                 <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-wrap-button" title="切换自动换行">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-expand-button" title="点击展开代码">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-copy-button" title="复制代码">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                </div>
               </div>
               <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
               </div>
               <div class="crayon-plain-wrap">
                <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
                 he.hadoop.mapreduce.TaskCounter instead
2015-06-20 16:28:54,901 ERROR [main]: exec.Task (SessionState.java:printError(545)) - Failed with exception Wrong FS: hdfs://server-01:8020/apps/hive/temp/temp_base, expected: hdfs://mycluster
java.lang.IllegalArgumentException: Wrong FS: hdfs://server-01:8020/apps/hive/temp/temp_base, expected: hdfs://mycluster
      at org.apache.hadoop.fs.FileSystem.checkPath(FileSystem.java:642)
      at org.apache.hadoop.hdfs.DistributedFileSystem.getPathName(DistributedFileSystem.java:181)
                </textarea>
               </div>
               <div class="crayon-main" style="">
                <table class="crayon-table">
                 <tbody>
                  <tr class="crayon-row">
                   <td class="crayon-nums " data-settings="show">
                    <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
                     <div class="crayon-num" data-line="crayon-57686a033d618513631619-1">
                      1
                     </div>
                     <div class="crayon-num crayon-striped-num" data-line="crayon-57686a033d618513631619-2">
                      2
                     </div>
                     <div class="crayon-num" data-line="crayon-57686a033d618513631619-3">
                      3
                     </div>
                     <div class="crayon-num crayon-striped-num" data-line="crayon-57686a033d618513631619-4">
                      4
                     </div>
                     <div class="crayon-num" data-line="crayon-57686a033d618513631619-5">
                      5
                     </div>
                    </div>
                   </td>
                   <td class="crayon-code">
                    <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
                     <div class="crayon-line" id="crayon-57686a033d618513631619-1">
                      <span class="crayon-v">
                       he
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       hadoop
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       mapreduce
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-e">
                       TaskCounter
                      </span>
                      <span class="crayon-i">
                       instead
                      </span>
                     </div>
                     <div class="crayon-line crayon-striped-line" id="crayon-57686a033d618513631619-2">
                      <span class="crayon-cn">
                       2015
                      </span>
                      <span class="crayon-o">
                       -
                      </span>
                      <span class="crayon-cn">
                       06
                      </span>
                      <span class="crayon-o">
                       -
                      </span>
                      <span class="crayon-cn">
                       20
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-cn">
                       16
                      </span>
                      <span class="crayon-o">
                       :
                      </span>
                      <span class="crayon-cn">
                       28
                      </span>
                      <span class="crayon-o">
                       :
                      </span>
                      <span class="crayon-cn">
                       54
                      </span>
                      <span class="crayon-sy">
                       ,
                      </span>
                      <span class="crayon-cn">
                       901
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-i">
                       ERROR
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-sy">
                       [
                      </span>
                      <span class="crayon-v">
                       main
                      </span>
                      <span class="crayon-sy">
                       ]
                      </span>
                      <span class="crayon-o">
                       :
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-v">
                       exec
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-e">
                       Task
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-sy">
                       (
                      </span>
                      <span class="crayon-v">
                       SessionState
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       java
                      </span>
                      <span class="crayon-o">
                       :
                      </span>
                      <span class="crayon-e">
                       printError
                      </span>
                      <span class="crayon-sy">
                       (
                      </span>
                      <span class="crayon-cn">
                       545
                      </span>
                      <span class="crayon-sy">
                       )
                      </span>
                      <span class="crayon-sy">
                       )
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-o">
                       -
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       Failed
                      </span>
                      <span class="crayon-e">
                       with
                      </span>
                      <span class="crayon-e">
                       exception
                      </span>
                      <span class="crayon-e">
                       Wrong
                      </span>
                      <span class="crayon-v">
                       FS
                      </span>
                      <span class="crayon-o">
                       :
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-v">
                       hdfs
                      </span>
                      <span class="crayon-o">
                       :
                      </span>
                      <span class="crayon-c">
                       //server-01:8020/apps/hive/temp/temp_base, expected: hdfs://mycluster
                      </span>
                     </div>
                     <div class="crayon-line" id="crayon-57686a033d618513631619-3">
                      <span class="crayon-v">
                       java
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       lang
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       IllegalArgumentException
                      </span>
                      <span class="crayon-o">
                       :
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       Wrong
                      </span>
                      <span class="crayon-v">
                       FS
                      </span>
                      <span class="crayon-o">
                       :
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-v">
                       hdfs
                      </span>
                      <span class="crayon-o">
                       :
                      </span>
                      <span class="crayon-c">
                       //server-01:8020/apps/hive/temp/temp_base, expected: hdfs://mycluster
                      </span>
                     </div>
                     <div class="crayon-line crayon-striped-line" id="crayon-57686a033d618513631619-4">
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       at
                      </span>
                      <span class="crayon-v">
                       org
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       apache
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       hadoop
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       fs
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       FileSystem
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-e">
                       checkPath
                      </span>
                      <span class="crayon-sy">
                       (
                      </span>
                      <span class="crayon-v">
                       FileSystem
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       java
                      </span>
                      <span class="crayon-o">
                       :
                      </span>
                      <span class="crayon-cn">
                       642
                      </span>
                      <span class="crayon-sy">
                       )
                      </span>
                     </div>
                     <div class="crayon-line" id="crayon-57686a033d618513631619-5">
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       at
                      </span>
                      <span class="crayon-v">
                       org
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       apache
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       hadoop
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       hdfs
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       DistributedFileSystem
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-e">
                       getPathName
                      </span>
                      <span class="crayon-sy">
                       (
                      </span>
                      <span class="crayon-v">
                       DistributedFileSystem
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       java
                      </span>
                      <span class="crayon-o">
                       :
                      </span>
                      <span class="crayon-cn">
                       181
                      </span>
                      <span class="crayon-sy">
                       )
                      </span>
                     </div>
                    </div>
                   </td>
                  </tr>
                 </tbody>
                </table>
               </div>
              </div>
              <!-- [Format Time: 0.0193 seconds] -->
              <p>
              </p>
             </td>
             <td class="code">
             </td>
            </tr>
           </tbody>
          </table>
         </figure>
        </li>
        <li>
         分析问题：
         <br/>
         <figure class="highlight plain">
          <table>
           <tbody>
            <tr>
             <td class="gutter">
             </td>
             <td class="code">
              <p>
              </p>
              <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
              <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-57686a033d61f242140815" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
               <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
                <span class="crayon-title">
                </span>
                <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
                 <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-wrap-button" title="切换自动换行">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-expand-button" title="点击展开代码">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-copy-button" title="复制代码">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                </div>
               </div>
               <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
               </div>
               <div class="crayon-plain-wrap">
                <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
                 &lt;span class="line"&gt;升级HA之后hive源数据信息hdfs地址问题，排查hive元数据信息发现，原来是hdfs路径
原来时某个节点非HA的地址，升级namenode为HA之后，地址并没有跟着变化，需要手动
修改hive 元数据信息比如存储在mysql中hdfs路径信息为HA的地址，可以解决。&lt;/span&gt;
                </textarea>
               </div>
               <div class="crayon-main" style="">
                <table class="crayon-table">
                 <tbody>
                  <tr class="crayon-row">
                   <td class="crayon-nums " data-settings="show">
                    <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
                     <div class="crayon-num" data-line="crayon-57686a033d61f242140815-1">
                      1
                     </div>
                     <div class="crayon-num crayon-striped-num" data-line="crayon-57686a033d61f242140815-2">
                      2
                     </div>
                     <div class="crayon-num" data-line="crayon-57686a033d61f242140815-3">
                      3
                     </div>
                    </div>
                   </td>
                   <td class="crayon-code">
                    <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
                     <div class="crayon-line" id="crayon-57686a033d61f242140815-1">
                      <span class="crayon-o">
                       &lt;
                      </span>
                      <span class="crayon-e">
                       span
                      </span>
                      <span class="crayon-t">
                       class
                      </span>
                      <span class="crayon-o">
                       =
                      </span>
                      <span class="crayon-s">
                       "line"
                      </span>
                      <span class="crayon-o">
                       &gt;
                      </span>
                      升级
                      <span class="crayon-i">
                       HA
                      </span>
                      之后
                      <span class="crayon-i">
                       hive
                      </span>
                      源数据信息
                      <span class="crayon-i">
                       hdfs
                      </span>
                      地址问题，排查
                      <span class="crayon-i">
                       hive
                      </span>
                      元数据信息发现，原来是
                      <span class="crayon-i">
                       hdfs
                      </span>
                      路径
                     </div>
                     <div class="crayon-line crayon-striped-line" id="crayon-57686a033d61f242140815-2">
                      原来时某个节点非
                      <span class="crayon-i">
                       HA
                      </span>
                      的地址，升级
                      <span class="crayon-i">
                       namenode
                      </span>
                      为
                      <span class="crayon-i">
                       HA
                      </span>
                      之后，地址并没有跟着变化，需要手动
                     </div>
                     <div class="crayon-line" id="crayon-57686a033d61f242140815-3">
                      修改
                      <span class="crayon-i">
                       hive
                      </span>
                      <span class="crayon-h">
                      </span>
                      元数据信息比如存储在
                      <span class="crayon-i">
                       mysql
                      </span>
                      中
                      <span class="crayon-i">
                       hdfs
                      </span>
                      路径信息为
                      <span class="crayon-i">
                       HA
                      </span>
                      的地址，可以解决。
                      <span class="crayon-o">
                       &lt;
                      </span>
                      <span class="crayon-o">
                       /
                      </span>
                      <span class="crayon-v">
                       span
                      </span>
                      <span class="crayon-o">
                       &gt;
                      </span>
                     </div>
                    </div>
                   </td>
                  </tr>
                 </tbody>
                </table>
               </div>
              </div>
              <!-- [Format Time: 0.0133 seconds] -->
              <p>
              </p>
             </td>
            </tr>
           </tbody>
          </table>
         </figure>
        </li>
       </ul>
       <h1 id="问题6：HDFS_ERROR_-_Failed_to_close_inode_330916">
        问题6：HDFS ERROR – Failed to close inode 330916
       </h1>
       <ul>
        <li>
         日志：
         <br/>
         <figure class="highlight plain">
          <table>
           <tbody>
            <tr>
             <td class="gutter">
             </td>
             <td class="code">
              <p>
              </p>
              <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
              <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-57686a033d625566591931" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
               <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
                <span class="crayon-title">
                </span>
                <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
                 <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-wrap-button" title="切换自动换行">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-expand-button" title="点击展开代码">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-copy-button" title="复制代码">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <span class="crayon-language">
                  Java
                 </span>
                </div>
               </div>
               <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
               </div>
               <div class="crayon-plain-wrap">
                <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
                 HDFS报错: Failed to close inode 330916
16/03/15 23:19:17 INFO util.HiveUtil: Run Hive Sql:MSCK REPAIR TABLE TB_INTERF_KW_TEMP
 16/03/15 23:19:18 ERROR hdfs.DFSClient: Failed to close inode 330916
 org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException): No lease on /tmp/abc/data/dictionary/KwTemp.txt (inode 330916): File does not exist. Holder DFSClient_NONMAPREDUCE_-1085844490_1 does not have any open files.
                </textarea>
               </div>
               <div class="crayon-main" style="">
                <table class="crayon-table">
                 <tbody>
                  <tr class="crayon-row">
                   <td class="crayon-nums " data-settings="show">
                    <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
                     <div class="crayon-num" data-line="crayon-57686a033d625566591931-1">
                      1
                     </div>
                     <div class="crayon-num crayon-striped-num" data-line="crayon-57686a033d625566591931-2">
                      2
                     </div>
                     <div class="crayon-num" data-line="crayon-57686a033d625566591931-3">
                      3
                     </div>
                     <div class="crayon-num crayon-striped-num" data-line="crayon-57686a033d625566591931-4">
                      4
                     </div>
                    </div>
                   </td>
                   <td class="crayon-code">
                    <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
                     <div class="crayon-line" id="crayon-57686a033d625566591931-1">
                      <span class="crayon-i">
                       HDFS
                      </span>
                      报错
                      <span class="crayon-o">
                       :
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       Failed
                      </span>
                      <span class="crayon-st">
                       to
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       close
                      </span>
                      <span class="crayon-i">
                       inode
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-cn">
                       330916
                      </span>
                     </div>
                     <div class="crayon-line crayon-striped-line" id="crayon-57686a033d625566591931-2">
                      <span class="crayon-cn">
                       16
                      </span>
                      <span class="crayon-o">
                       /
                      </span>
                      <span class="crayon-cn">
                       03
                      </span>
                      <span class="crayon-o">
                       /
                      </span>
                      <span class="crayon-cn">
                       15
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-cn">
                       23
                      </span>
                      <span class="crayon-o">
                       :
                      </span>
                      <span class="crayon-cn">
                       19
                      </span>
                      <span class="crayon-o">
                       :
                      </span>
                      <span class="crayon-cn">
                       17
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       INFO
                      </span>
                      <span class="crayon-v">
                       util
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       HiveUtil
                      </span>
                      <span class="crayon-o">
                       :
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       Run
                      </span>
                      <span class="crayon-e">
                       Hive
                      </span>
                      <span class="crayon-v">
                       Sql
                      </span>
                      <span class="crayon-o">
                       :
                      </span>
                      <span class="crayon-e">
                       MSCK
                      </span>
                      <span class="crayon-e">
                       REPAIR
                      </span>
                      <span class="crayon-e">
                       TABLE
                      </span>
                      <span class="crayon-v">
                       TB_INTERF_KW
                      </span>
                      <span class="crayon-sy">
                       _
                      </span>
                      TEMP
                     </div>
                     <div class="crayon-line" id="crayon-57686a033d625566591931-3">
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-cn">
                       16
                      </span>
                      <span class="crayon-o">
                       /
                      </span>
                      <span class="crayon-cn">
                       03
                      </span>
                      <span class="crayon-o">
                       /
                      </span>
                      <span class="crayon-cn">
                       15
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-cn">
                       23
                      </span>
                      <span class="crayon-o">
                       :
                      </span>
                      <span class="crayon-cn">
                       19
                      </span>
                      <span class="crayon-o">
                       :
                      </span>
                      <span class="crayon-cn">
                       18
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       ERROR
                      </span>
                      <span class="crayon-v">
                       hdfs
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       DFSClient
                      </span>
                      <span class="crayon-o">
                       :
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       Failed
                      </span>
                      <span class="crayon-st">
                       to
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       close
                      </span>
                      <span class="crayon-i">
                       inode
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-cn">
                       330916
                      </span>
                     </div>
                     <div class="crayon-line crayon-striped-line" id="crayon-57686a033d625566591931-4">
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-v">
                       org
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       apache
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       hadoop
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       ipc
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-e">
                       RemoteException
                      </span>
                      <span class="crayon-sy">
                       (
                      </span>
                      <span class="crayon-v">
                       org
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       apache
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       hadoop
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       hdfs
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       server
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       namenode
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-v">
                       LeaseExpiredException
                      </span>
                      <span class="crayon-sy">
                       )
                      </span>
                      <span class="crayon-o">
                       :
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       No
                      </span>
                      <span class="crayon-e">
                       lease
                      </span>
                      <span class="crayon-v">
                       on
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-o">
                       /
                      </span>
                      <span class="crayon-v">
                       tmp
                      </span>
                      <span class="crayon-o">
                       /
                      </span>
                      <span class="crayon-v">
                       abc
                      </span>
                      <span class="crayon-o">
                       /
                      </span>
                      <span class="crayon-v">
                       data
                      </span>
                      <span class="crayon-o">
                       /
                      </span>
                      <span class="crayon-v">
                       dictionary
                      </span>
                      <span class="crayon-o">
                       /
                      </span>
                      <span class="crayon-v">
                       KwTemp
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-e">
                       txt
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-sy">
                       (
                      </span>
                      <span class="crayon-i">
                       inode
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-cn">
                       330916
                      </span>
                      <span class="crayon-sy">
                       )
                      </span>
                      <span class="crayon-o">
                       :
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       File
                      </span>
                      <span class="crayon-e">
                       does
                      </span>
                      <span class="crayon-st">
                       not
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-v">
                       exist
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       Holder
                      </span>
                      <span class="crayon-v">
                       DFSClient_NONMAPREDUCE_
                      </span>
                      <span class="crayon-o">
                       -
                      </span>
                      <span class="crayon-cn">
                       1085844490_1
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       does
                      </span>
                      <span class="crayon-st">
                       not
                      </span>
                      <span class="crayon-h">
                      </span>
                      <span class="crayon-e">
                       have
                      </span>
                      <span class="crayon-e">
                       any
                      </span>
                      <span class="crayon-e">
                       open
                      </span>
                      <span class="crayon-v">
                       files
                      </span>
                      <span class="crayon-sy">
                       .
                      </span>
                     </div>
                    </div>
                   </td>
                  </tr>
                 </tbody>
                </table>
               </div>
              </div>
              <!-- [Format Time: 0.0133 seconds] -->
              <p>
              </p>
             </td>
            </tr>
           </tbody>
          </table>
         </figure>
        </li>
        <li>
         分析问题：
         <br/>
         <figure class="highlight plain">
          <table>
           <tbody>
            <tr>
             <td class="gutter">
             </td>
             <td class="code">
              <p>
              </p>
              <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
              <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-57686a033d62c712852733" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
               <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
                <span class="crayon-title">
                </span>
                <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
                 <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-wrap-button" title="切换自动换行">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-expand-button" title="点击展开代码">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-copy-button" title="复制代码">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                 <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
                  <div class="crayon-button-icon">
                  </div>
                 </div>
                </div>
               </div>
               <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
               </div>
               <div class="crayon-plain-wrap">
                <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
                 &lt;span class="line"&gt;程序使用HDFS api操作的时候，在mr中操作，判断一个目录是否存在，而导致的问题，总之是API和集群沟通问题。&lt;/span&gt;
                </textarea>
               </div>
               <div class="crayon-main" style="">
                <table class="crayon-table">
                 <tbody>
                  <tr class="crayon-row">
                   <td class="crayon-nums " data-settings="show">
                    <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
                     <div class="crayon-num" data-line="crayon-57686a033d62c712852733-1">
                      1
                     </div>
                    </div>
                   </td>
                   <td class="crayon-code">
                    <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
                     <div class="crayon-line" id="crayon-57686a033d62c712852733-1">
                      <span class="crayon-o">
                       &lt;
                      </span>
                      <span class="crayon-e">
                       span
                      </span>
                      <span class="crayon-t">
                       class
                      </span>
                      <span class="crayon-o">
                       =
                      </span>
                      <span class="crayon-s">
                       "line"
                      </span>
                      <span class="crayon-o">
                       &gt;
                      </span>
                      程序使用
                      <span class="crayon-e">
                       HDFS
                      </span>
                      <span class="crayon-i">
                       api
                      </span>
                      操作的时候，在
                      <span class="crayon-i">
                       mr
                      </span>
                      中操作，判断一个目录是否存在，而导致的问题，总之是
                      <span class="crayon-i">
                       API
                      </span>
                      和集群沟通问题。
                      <span class="crayon-o">
                       &lt;
                      </span>
                      <span class="crayon-o">
                       /
                      </span>
                      <span class="crayon-v">
                       span
                      </span>
                      <span class="crayon-o">
                       &gt;
                      </span>
                     </div>
                    </div>
                   </td>
                  </tr>
                 </tbody>
                </table>
               </div>
              </div>
              <!-- [Format Time: 0.0014 seconds] -->
              <p>
              </p>
             </td>
            </tr>
           </tbody>
          </table>
         </figure>
        </li>
       </ul>
       <p>
        原创文章，转载请注明： 转载自
        <a href="http://www.itweet.cn/">
         whoami
        </a>
        的博客
       </p>
       <p>
       </p>
       <p>
        原文链接：
        <a href="http://www.itweet.cn/2016/03/17/Hadoop-issue-collection/?hmsr=toutiao.io&amp;utm_medium=toutiao.io&amp;utm_source=toutiao.io">
         http://www.itweet.cn/2016/03/17/Hadoop-issue-collection/?hmsr=toutiao.io&amp;utm_medium=toutiao.io&amp;utm_source=toutiao.io
        </a>
       </p>
      </div>
      <div>
       <strong>
        注：转载文章均来自于公开网络，仅供学习使用，不会用于任何商业用途，如果侵犯到原作者的权益，请您与我们联系删除或者授权事宜，联系邮箱：contact@dataunion.org。转载数盟网站文章请注明原文章作者，否则产生的任何版权纠纷与数盟无关。
       </strong>
      </div>
      <!--content_text-->
      <div class="fenxian">
       <!-- JiaThis Button BEGIN -->
       <div class="jiathis_style_32x32">
        <p class="jiathis_button_weixin">
        </p>
        <p class="jiathis_button_tsina">
        </p>
        <p class="jiathis_button_qzone">
        </p>
        <p class="jiathis_button_cqq">
        </p>
        <p class="jiathis_button_tumblr">
        </p>
        <a class="jiathis jiathis_txt jtico jtico_jiathis" href="http://www.jiathis.com/share" target="_blank">
        </a>
        <p class="jiathis_counter_style">
        </p>
       </div>
       <!-- JiaThis Button END -->
      </div>
     </article>
     <!--content-->
     <!--相关文章-->
     <div class="xianguan">
      <div class="xianguantitle">
       相关文章！
      </div>
      <ul class="pic">
       <li>
        <a href="http://dataunion.org/24678.html">
         <img src="http://dataunion.org/wp-content/uploads/2016/06/20140917125452915416-216x200.jpg"/>
        </a>
        <a class="link" href="http://dataunion.org/24678.html" rel="bookmark" title="python3中的正则模块">
         python3中的正则模块
        </a>
       </li>
       <li>
        <a href="http://dataunion.org/24675.html">
         <img src="http://dataunion.org/wp-content/uploads/2016/06/t015b337bd75d9ef893-161x200.jpg"/>
        </a>
        <a class="link" href="http://dataunion.org/24675.html" rel="bookmark" title="注释是恶魔，请不要再写一行注释">
         注释是恶魔，请不要再写一行注释
        </a>
       </li>
       <li>
        <a href="http://dataunion.org/24660.html">
         <img src="http://dataunion.org/wp-content/uploads/2016/06/u16130037972892789947fm21gp0-300x157.jpg"/>
        </a>
        <a class="link" href="http://dataunion.org/24660.html" rel="bookmark" title="如何从Github上轻松安装R包">
         如何从Github上轻松安装R包
        </a>
       </li>
       <li>
        <a href="http://dataunion.org/24654.html">
         <img src="http://dataunion.org/wp-content/uploads/2016/06/df53fac99fc53ba5a90666abcca25e6d_b-267x200.png"/>
        </a>
        <a class="link" href="http://dataunion.org/24654.html" rel="bookmark" title="简单形象又有趣地说说强大的神经网络">
         简单形象又有趣地说说强大的神经网络
        </a>
       </li>
      </ul>
     </div>
     <!--相关文章-->
     <div class="comment" id="comments">
      <!-- You can start editing here. -->
      <!-- If comments are open, but there are no comments. -->
      <div class="title">
       期待你一针见血的评论，Come on！
      </div>
      <div id="respond">
       <p>
        不用想啦，马上
        <a href="http://dataunion.org/wp-login.php?redirect_to=http%3A%2F%2Fdataunion.org%2F22887.html">
         "登录"
        </a>
        发表自已的想法.
       </p>
      </div>
     </div>
     <!-- .nav-single -->
    </div>
    <!--Container End-->
    <aside id="sitebar">
     <div class="sitebar_list2">
      <div class="wptag">
       <span class="tagtitle">
        热门标签+
       </span>
       <div class="tagg">
        <ul class="menu" id="menu-%e5%8f%8b%e6%83%85%e9%93%be%e6%8e%a5">
         <li class="menu-item menu-item-type-custom menu-item-object-custom menu-item-1605" id="menu-item-1605">
          <a href="http://taidizh.com/">
           泰迪智慧
          </a>
         </li>
         <li class="menu-item menu-item-type-custom menu-item-object-custom menu-item-20884" id="menu-item-20884">
          <a href="http://www.transwarp.cn/">
           星环科技
          </a>
         </li>
         <li class="menu-item menu-item-type-custom menu-item-object-custom menu-item-3538" id="menu-item-3538">
          <a href="http://datall.org/">
           珈和遥感
          </a>
         </li>
         <li class="menu-item menu-item-type-custom menu-item-object-custom menu-item-20888" id="menu-item-20888">
          <a href="http://www.chinahadoop.cn/">
           小象学院
          </a>
         </li>
        </ul>
       </div>
      </div>
     </div>
     <div class="sitebar_list">
      <div class="textwidget">
       <div align="center">
        <a href="http://study.163.com/course/courseMain.htm?courseId=991022" target="_blank">
         <img src="http://dataunion.org/wp-content/uploads/2016/03/dv.jpg"/>
        </a>
       </div>
      </div>
     </div>
     <div class="sitebar_list">
      <h4 class="sitebar_title">
       文章分类
      </h4>
      <div class="tagcloud">
       <a class="tag-link-44" href="http://dataunion.org/category/industry/demo" style="font-size: 10.204724409449pt;" title="4个话题">
        Demo展示
       </a>
       <a class="tag-link-31" href="http://dataunion.org/category/experts" style="font-size: 15.826771653543pt;" title="52个话题">
        专家团队
       </a>
       <a class="tag-link-870" href="http://dataunion.org/category/tech/ai" style="font-size: 19.795275590551pt;" title="273个话题">
        人工智能
       </a>
       <a class="tag-link-488" href="http://dataunion.org/category/%e5%8a%a0%e5%85%a5%e6%95%b0%e7%9b%9f" style="font-size: 8pt;" title="1个话题">
        加入数盟
       </a>
       <a class="tag-link-869" href="http://dataunion.org/category/tech/viz" style="font-size: 17.204724409449pt;" title="93个话题">
        可视化
       </a>
       <a class="tag-link-30" href="http://dataunion.org/category/partners" style="font-size: 10.645669291339pt;" title="5个话题">
        合作伙伴
       </a>
       <a class="tag-link-889" href="http://dataunion.org/category/parterc" style="font-size: 11.582677165354pt;" title="8个话题">
        合作会议
       </a>
       <a class="tag-link-104" href="http://dataunion.org/category/books" style="font-size: 12.96062992126pt;" title="15个话题">
        图书
       </a>
       <a class="tag-link-220" href="http://dataunion.org/category/tech/base" style="font-size: 19.850393700787pt;" title="281个话题">
        基础架构
       </a>
       <a class="tag-link-219" href="http://dataunion.org/category/tech/analysis" style="font-size: 19.409448818898pt;" title="232个话题">
        数据分析
       </a>
       <a class="tag-link-887" href="http://dataunion.org/category/tech/dm" style="font-size: 13.291338582677pt;" title="17个话题">
        数据挖掘
       </a>
       <a class="tag-link-34" href="http://dataunion.org/category/tech" style="font-size: 20.732283464567pt;" title="404个话题">
        文章
       </a>
       <a class="tag-link-1" href="http://dataunion.org/category/uncategorized" style="font-size: 22pt;" title="693个话题">
        未分类
       </a>
       <a class="tag-link-4" href="http://dataunion.org/category/events" style="font-size: 14.503937007874pt;" title="29个话题">
        活动
       </a>
       <a class="tag-link-890" href="http://dataunion.org/category/tech/%e6%b7%b1%e5%ba%a6%e5%ad%a6%e4%b9%a0" style="font-size: 10.204724409449pt;" title="4个话题">
        深度学习
       </a>
       <a class="tag-link-221" href="http://dataunion.org/category/tech/devl" style="font-size: 18.968503937008pt;" title="193个话题">
        编程语言
       </a>
       <a class="tag-link-888" href="http://dataunion.org/category/career" style="font-size: 15.661417322835pt;" title="48个话题">
        职业规划
       </a>
       <a class="tag-link-5" href="http://dataunion.org/category/jobs" style="font-size: 14.11811023622pt;" title="25个话题">
        职位
       </a>
       <a class="tag-link-871" href="http://dataunion.org/category/industry" style="font-size: 15.716535433071pt;" title="49个话题">
        行业
       </a>
       <a class="tag-link-613" href="http://dataunion.org/category/industry/case" style="font-size: 16.984251968504pt;" title="84个话题">
        行业应用
       </a>
       <a class="tag-link-885" href="http://dataunion.org/category/industry/news" style="font-size: 17.425196850394pt;" title="102个话题">
        行业资讯
       </a>
       <a class="tag-link-10" href="http://dataunion.org/category/training" style="font-size: 14.228346456693pt;" title="26个话题">
        课程
       </a>
       <a class="tag-link-16" href="http://dataunion.org/category/sources" style="font-size: 15.661417322835pt;" title="48个话题">
        资源
       </a>
      </div>
     </div>
     <div class="sitebar_list">
      <h4 class="sitebar_title">
       功能
      </h4>
      <ul>
       <li>
        <a href="http://dataunion.org/wp-login.php?action=register">
         注册
        </a>
       </li>
       <li>
        <a href="http://dataunion.org/wp-login.php">
         登录
        </a>
       </li>
       <li>
        <a href="http://dataunion.org/feed">
         文章
         <abbr title="Really Simple Syndication">
          RSS
         </abbr>
        </a>
       </li>
       <li>
        <a href="http://dataunion.org/comments/feed">
         评论
         <abbr title="Really Simple Syndication">
          RSS
         </abbr>
        </a>
       </li>
       <li>
        <a href="https://cn.wordpress.org/" title="基于WordPress，一个优美、先进的个人信息发布平台。">
         WordPress.org
        </a>
       </li>
      </ul>
     </div>
    </aside>
    <div class="clear">
    </div>
   </div>
   <!--main-->
   ﻿
   <footer id="dibu">
    <div class="about">
     <div class="right">
      <ul class="menu" id="menu-%e5%ba%95%e9%83%a8%e8%8f%9c%e5%8d%95">
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-18024" id="menu-item-18024">
        <a href="http://dataunion.org/category/partners">
         合作伙伴
        </a>
       </li>
       <li class="menu-item menu-item-type-post_type menu-item-object-page menu-item-20881" id="menu-item-20881">
        <a href="http://dataunion.org/contribute">
         文章投稿
        </a>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-20872" id="menu-item-20872">
        <a href="http://dataunion.org/category/%e5%8a%a0%e5%85%a5%e6%95%b0%e7%9b%9f">
         加入数盟
        </a>
       </li>
       <li class="menu-item menu-item-type-post_type menu-item-object-page menu-item-22441" id="menu-item-22441">
        <a href="http://dataunion.org/f-links">
         友情链接
        </a>
       </li>
       <li class="menu-item menu-item-type-post_type menu-item-object-page menu-item-20874" id="menu-item-20874">
        <a href="http://dataunion.org/aboutus">
         关于数盟
        </a>
       </li>
      </ul>
      <p class="banquan">
       数盟社区        ，
        做最棒的数据科学社区
      </p>
     </div>
     <div class="left">
      <ul class="bottomlist">
       <li>
        <a href="http://weibo.com/DataScientistUnion  " target="_blank" 　title="">
         <img src="http://dataunion.org/wp-content/themes/yzipi/images/weibo.png"/>
        </a>
       </li>
       <li>
        <a class="cd-popup-trigger" href="http://dataunion.org/22887.html#0">
         <img src="http://dataunion.org/wp-content/themes/yzipi/images/weixin.png"/>
        </a>
       </li>
      </ul>
      <div class="cd-popup">
       <div class="cd-popup-container">
        <h1>
         扫描二维码,加微信公众号
        </h1>
        <img src="http://dataunion.org/wp-content/themes/yzipi/images/2014-12-06-1515289049.png"/>
        <a class="cd-popup-close" href="http://dataunion.org/22887.html">
        </a>
       </div>
       <!-- cd-popup-container -->
      </div>
      <!-- cd-popup -->
     </div>
    </div>
    <!--about-->
    <div class="bottom">
     <a href="http://dataunion.org/">
      数盟社区
     </a>
     <a href="http://www.miitbeian.gov.cn/" rel="external nofollow" target="_blank">
      京ICP备14026740号
     </a>
     联系我们：
     <a href="mailto:contact@dataunion.org" target="_blank">
      contact@dataunion.org
     </a>
     <div class="tongji">
     </div>
     <!--bottom-->
     <div class="scroll" id="scroll" style="display:none;">
      ︿
     </div>
    </div>
   </footer>
   <!--dibu-->
  </div>
 </body>
</html>